import requests as res
from bs4 import BeautifulSoup as bs
from urllib import parse
import time
from dateutil import parser
from pymongo import MongoClient
import bson
from lxml import etree
import re
import argparse 
import json
# from to_content import get_content

def code(s):
    chinese = s.encode("utf-8")
    urlcode = parse.quote(chinese)
    return urlcode
def find_true_url(html):
    n_url = ''
    parts = re.findall(r'\+\=\ \'.+', html)

    for i in parts:
        n_url += i[4:-3]
    n_url.replace("@", "")
    return n_url

#link为文章的weixin.sougou链接
def get_content(link,cookies):
    header = {

    #'Cookie': 'IPLOC=CN1100; SUV=1637390049995640; SUID=D5D79B271431A40A00000000619896CB; ABTEST=2|1637390032|v1; weixinIndexVisited=1; LCLKINT=8691; LSTMV=263%2C186; SNUID=A98F595A2025F19B458BEAC720609F0B; JSESSIONID=aaa5b6ZJYYiw-acMFmf0x; ppinf=5|1638071795|1639281395|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo5OiVFMyU4MCU4MnxjcnQ6MTA6MTYzODA3MTc5NXxyZWZuaWNrOjk6JUUzJTgwJTgyfHVzZXJpZDo0NDpvOXQybHVMYVpJeWlKNTZlZTExdUJob1hxcU04QHdlaXhpbi5zb2h1LmNvbXw; pprdig=I8ZOSkkloGUh7xc9_9tvgbQXq4NOtTAoyEpUMlp3beeVC0KTFPAQKuE1cyHLIVvNyPTJUZ30PDde2b29uZ-p7TbbMVs3FSrEMZKxwKW2v66udAGegJ-WNfToRjMeKZEbsub0f_ar14EaP__gKBu1QPjxdN3SVcWJZASSSmGW-EA; ppinfo=85466e9fe5; passport=5|1638071795|1639281395|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo5OiVFMyU4MCU4MnxjcnQ6MTA6MTYzODA3MTc5NXxyZWZuaWNrOjk6JUUzJTgwJTgyfHVzZXJpZDo0NDpvOXQybHVMYVpJeWlKNTZlZTExdUJob1hxcU04QHdlaXhpbi5zb2h1LmNvbXw|8d07c0e05a|I8ZOSkkloGUh7xc9_9tvgbQXq4NOtTAoyEpUMlp3beeVC0KTFPAQKuE1cyHLIVvNyPTJUZ30PDde2b29uZ-p7TbbMVs3FSrEMZKxwKW2v66udAGegJ-WNfToRjMeKZEbsub0f_ar14EaP__gKBu1QPjxdN3SVcWJZASSSmGW-EA; sgid=08-54235003-AWGiaicfNiccPzYicZ6c2PicHzek; ppmdig=1638071796000000907592a7d2fcdb307f22be28bcba3443',
    #'Cookie':'IPLOC=CN1100; SUID=7229F73C1431A40A0000000061923A54; SUV=1636973141936444; ABTEST=0|1637238601|v1; weixinIndexVisited=1; ld=5lllllllll2P60IUlllllppYXN7lllll55tItkllll9lllllVllll5@@@@@@@@@@; SNUID=5F7EA06B585D856DCE63ED75585D8D45; ppinf=5|1638502558|1639712158|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo0ODpneG0lRTQlQkIlOEElRTUlQTQlQTklRTUlOTAlODMlRTQlQkIlODAlRTQlQjklODh8Y3J0OjEwOjE2Mzg1MDI1NTh8cmVmbmljazo0ODpneG0lRTQlQkIlOEElRTUlQTQlQTklRTUlOTAlODMlRTQlQkIlODAlRTQlQjklODh8dXNlcmlkOjQ0Om85dDJsdU43Ql9wdE9DTEQxSjRxSzJla1dwQmNAd2VpeGluLnNvaHUuY29tfA; pprdig=XN5aZp6kN_higgOeqKVpNgMP0A0WeqCdz4G-1Tlqh2CZBC2cDFhm9EvSeEuzrOMZw_lIbIWsFSpetlyyviNutRr4lvJWJwg60VGprqv49rUVaKhnGu13QWXjitFyc012AjUxGTcitTQnkFImspm0h3QC3-dnKtPck_w3V-vgPjk; ppinfo=f440c19c8d; passport=5|1638502558|1639712158|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo0ODpneG0lRTQlQkIlOEElRTUlQTQlQTklRTUlOTAlODMlRTQlQkIlODAlRTQlQjklODh8Y3J0OjEwOjE2Mzg1MDI1NTh8cmVmbmljazo0ODpneG0lRTQlQkIlOEElRTUlQTQlQTklRTUlOTAlODMlRTQlQkIlODAlRTQlQjklODh8dXNlcmlkOjQ0Om85dDJsdU43Ql9wdE9DTEQxSjRxSzJla1dwQmNAd2VpeGluLnNvaHUuY29tfA|ad716ac8df|XN5aZp6kN_higgOeqKVpNgMP0A0WeqCdz4G-1Tlqh2CZBC2cDFhm9EvSeEuzrOMZw_lIbIWsFSpetlyyviNutRr4lvJWJwg60VGprqv49rUVaKhnGu13QWXjitFyc012AjUxGTcitTQnkFImspm0h3QC3-dnKtPck_w3V-vgPjk; sgid=05-52078565-AWGpkJ7cBLQlMCNGmoswMLU; JSESSIONID=aaa0USzdlnNHo6KxlCQ1x; ppmdig=1638523489000000d5986846f2dffecaa098713dbeb50232',
    'Cookie':cookies

    } 


    web1 = res.get(link,headers=header)
    web1.encoding = web1.apparent_encoding
    try:
        url = find_true_url(web1.text)
        # print(url)
        web = res.get(url)
        web.encoding = web.apparent_encoding
        soup = bs(web.text,"lxml")
        text = soup.select("#page-content > .rich_media_area_primary_inner > #img-content > .rich_media_content")[0]
        # print(text)
        # time.sleep(random.randint(40,80))
        return str(text.get_text())
    except:
        print("content为空")
        return ""

def main():
    parser = argparse.ArgumentParser(description="RUN WEIXINSPIDER")
    # parser.add_argument('-c','--cookie')
    # parser.add_argument('-k','--keyword')
    # parser.add_argument('-n','--collection_name')
    parser.add_argument('-i','--id')
    args =  parser.parse_args()
    id = args.id
    mongourl = "mongodb://yuqing:Bigdata#2021@175.27.160.162:27017"
    client = MongoClient(mongourl)
    db = client["Task"]
    collection_task = db['task']
    condition = {'_id':id}
    task = collection_task.find_one(condition)
    info = task['canshu']
    info_json = json.loads(info)
    cookie = info_json['cookies']
    collection_name = info_json['dbname']
    keyword = info_json['kwd']
    wxspider(cookie, keyword, collection_name,id)

def wxspider(cookies,keywords,collection_name,id):
    headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Referer': 'https://open.weixin.qq.com/',
    'Cache-Control': 'max-age=0',
    'Connection': 'keep-alive',
    #'Cookie': 'IPLOC=CN1100; SUV=1637390049995640; SUID=D5D79B271431A40A00000000619896CB; ABTEST=2|1637390032|v1; weixinIndexVisited=1; LCLKINT=8691; LSTMV=263%2C186; SNUID=A98F595A2025F19B458BEAC720609F0B; JSESSIONID=aaa5b6ZJYYiw-acMFmf0x; ppinf=5|1638071795|1639281395|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo5OiVFMyU4MCU4MnxjcnQ6MTA6MTYzODA3MTc5NXxyZWZuaWNrOjk6JUUzJTgwJTgyfHVzZXJpZDo0NDpvOXQybHVMYVpJeWlKNTZlZTExdUJob1hxcU04QHdlaXhpbi5zb2h1LmNvbXw; pprdig=I8ZOSkkloGUh7xc9_9tvgbQXq4NOtTAoyEpUMlp3beeVC0KTFPAQKuE1cyHLIVvNyPTJUZ30PDde2b29uZ-p7TbbMVs3FSrEMZKxwKW2v66udAGegJ-WNfToRjMeKZEbsub0f_ar14EaP__gKBu1QPjxdN3SVcWJZASSSmGW-EA; ppinfo=85466e9fe5; passport=5|1638071795|1639281395|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo5OiVFMyU4MCU4MnxjcnQ6MTA6MTYzODA3MTc5NXxyZWZuaWNrOjk6JUUzJTgwJTgyfHVzZXJpZDo0NDpvOXQybHVMYVpJeWlKNTZlZTExdUJob1hxcU04QHdlaXhpbi5zb2h1LmNvbXw|8d07c0e05a|I8ZOSkkloGUh7xc9_9tvgbQXq4NOtTAoyEpUMlp3beeVC0KTFPAQKuE1cyHLIVvNyPTJUZ30PDde2b29uZ-p7TbbMVs3FSrEMZKxwKW2v66udAGegJ-WNfToRjMeKZEbsub0f_ar14EaP__gKBu1QPjxdN3SVcWJZASSSmGW-EA; sgid=08-54235003-AWGiaicfNiccPzYicZ6c2PicHzek; ppmdig=1638071796000000907592a7d2fcdb307f22be28bcba3443',
    #'Cookie':'IPLOC=CN1100; SUID=7229F73C1431A40A0000000061923A54; SUV=1636973141936444; ABTEST=0|1637238601|v1; weixinIndexVisited=1; ld=5lllllllll2P60IUlllllppYXN7lllll55tItkllll9lllllVllll5@@@@@@@@@@; SNUID=5F7EA06B585D856DCE63ED75585D8D45; ppinf=5|1638502558|1639712158|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo0ODpneG0lRTQlQkIlOEElRTUlQTQlQTklRTUlOTAlODMlRTQlQkIlODAlRTQlQjklODh8Y3J0OjEwOjE2Mzg1MDI1NTh8cmVmbmljazo0ODpneG0lRTQlQkIlOEElRTUlQTQlQTklRTUlOTAlODMlRTQlQkIlODAlRTQlQjklODh8dXNlcmlkOjQ0Om85dDJsdU43Ql9wdE9DTEQxSjRxSzJla1dwQmNAd2VpeGluLnNvaHUuY29tfA; pprdig=XN5aZp6kN_higgOeqKVpNgMP0A0WeqCdz4G-1Tlqh2CZBC2cDFhm9EvSeEuzrOMZw_lIbIWsFSpetlyyviNutRr4lvJWJwg60VGprqv49rUVaKhnGu13QWXjitFyc012AjUxGTcitTQnkFImspm0h3QC3-dnKtPck_w3V-vgPjk; ppinfo=f440c19c8d; passport=5|1638502558|1639712158|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo0ODpneG0lRTQlQkIlOEElRTUlQTQlQTklRTUlOTAlODMlRTQlQkIlODAlRTQlQjklODh8Y3J0OjEwOjE2Mzg1MDI1NTh8cmVmbmljazo0ODpneG0lRTQlQkIlOEElRTUlQTQlQTklRTUlOTAlODMlRTQlQkIlODAlRTQlQjklODh8dXNlcmlkOjQ0Om85dDJsdU43Ql9wdE9DTEQxSjRxSzJla1dwQmNAd2VpeGluLnNvaHUuY29tfA|ad716ac8df|XN5aZp6kN_higgOeqKVpNgMP0A0WeqCdz4G-1Tlqh2CZBC2cDFhm9EvSeEuzrOMZw_lIbIWsFSpetlyyviNutRr4lvJWJwg60VGprqv49rUVaKhnGu13QWXjitFyc012AjUxGTcitTQnkFImspm0h3QC3-dnKtPck_w3V-vgPjk; sgid=05-52078565-AWGpkJ7cBLQlMCNGmoswMLU; JSESSIONID=aaa0USzdlnNHo6KxlCQ1x; ppmdig=1638523489000000d5986846f2dffecaa098713dbeb50232',
    'Cookie':cookies,
    'Host': 'weixin.sogou.com',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36'
    } 

    # keyword = ["毒跑道","袁贵仁：绝不能让传播西方价值观念教材进课堂","高考减招","中小学教材删除毛泽东鲁迅等作品"]
    # collection_name = ["doc_dupaodao","doc_WesternValuesTextbooks","doc_gaokaojianzhao","doc_DeleteArticles"]

    keyword = [f'{keywords}']
    # collection_name = ["doc_PuZhiFenLiu"]

    mongourl = "mongodb://yuqing:Bigdata#2021@175.27.160.162:27017"
    # mongourl = "mongodb://yuqing:Bigdata#2021@118.195.192.186:27017"
    # mongourl = "mongodb://localhost:27017"
    client = MongoClient(mongourl)
    db = client["Task"]
    collection = db[collection_name]
    collection_task = db['task']
    # task = {
    #     'todate':'2021-10-01',
    #     'fromdate':'2020-10-01',
    #     'completed_num':0,
    #     'keyword':keywords,
    #     'savetime':1638696433746
    # }
    # collection_task.insert_one(task)
    # collection = client["test"][collection_name[keyword.index(k)]]
    insert_count = 0
    all_time = []
    with collection.find({"channel": "weixin"}, no_cursor_timeout=True) as fr:
        for item in fr:
            all_time.append(int(time.mktime(time.strptime(str(item["time"]), '%Y-%m-%d %H:%M:%S'))))
    print("ok")
    condition = {'_id':id}
    result = collection_task.update_one(condition,{'$set':{'status':'run'}})
    print(result.matched_count,result.modified_count)

    for k in keyword:
        for j in range(1,101):

            url = "https://weixin.sogou.com/weixin?query={}&_sug_type_=&s_from=input&_sug_=y&type=2&page={}&ie=utf8".format(code(k),j)
            print(url)

            web = res.get(url,headers=headers)
            web.encoding = web.apparent_encoding
            soup = bs(web.text,"lxml")
            # print(soup)   
            title=soup.select(".main-left > div.news-box > ul >li > .txt-box > h3 > a") #标题
            dec = soup.select(".main-left > div.news-box > ul >li > .txt-box > p") #内容
            source = soup.select(".main-left > div.news-box > ul >li > .txt-box > .s-p > a") #来源
            date = soup.select(".main-left > div.news-box > ul >li > .txt-box > .s-p > .s2 >script") #时间
            # print(dec)
            # print(source)
            #
            for datetime in date:
                print(datetime)
            
            data = []
            error_count = 0
            print(title)
            if len(title) == 0:
                print("关键字{}的第{}页出错".format(k,j))
                client.close()
                break 
            
            
            for i in range(0,len(title)):
                try:
                    item = {}
                    shijian = date[i].text.split("'")[-2]
                    # print(shijian)
                    if shijian in all_time:
                        print("已存在")
                        continue
                    item["time"] = parser.parse(time.strftime('%Y-%m-%dT%H:%M:%S.000+00:00', time.localtime(float(shijian))))
                    item["title"] = title[i].text
                    print(title[i].text)
                    link = "https://weixin.sogou.com"+title[i]["href"]
                    item["_id"] = link
                    item["content"] = get_content(link,cookies)
                    item["source"] = source[i].text
                    item["savetime"] = bson.int64.Int64(time.time())
                    item["channel"] = "weixin"
                    item["subject"] = k


                    data.append(item)
                except:
                    print("数据查询出错")
                    error_count += 1
                    if error_count >= 10:
                        client.close()
                        # time.sleep(random.randint(40,70))
                        break
                    continue
            try:
                collection.insert_many(data)
                print("关键字{}的第{}页ok".format(k,j))
                insert_count += 10
                result = collection_task.update_one(condition,{'$set':{'complete':insert_count}})
                print(result.matched_count,result.modified_count)

            except Exception as e:
                print(e)
                continue
            # return insert_count

            data=[]
            # client.close()
            # time.sleep(random.randint(40,80))
    now_time = int(round(time.time()*1000))
    collection_task.update_one(condition,{'$set':{'savetime':now_time}})        
    print('已完成全部任务')
    client.close()
    
if __name__ == "__main__":
#     wxspider('IPLOC=CN1100; SUID=7229F73C1431A40A0000000061923A54; SUV=1636973141936444; ABTEST=0|1637238601|v1; weixinIndexVisited=1; ld=5lllllllll2P60IUlllllppYXN7lllll55tItkllll9lllllVllll5@@@@@@@@@@; SNUID=5F7EA06B585D856DCE63ED75585D8D45; ppinf=5|1638502558|1639712158|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo0ODpneG0lRTQlQkIlOEElRTUlQTQlQTklRTUlOTAlODMlRTQlQkIlODAlRTQlQjklODh8Y3J0OjEwOjE2Mzg1MDI1NTh8cmVmbmljazo0ODpneG0lRTQlQkIlOEElRTUlQTQlQTklRTUlOTAlODMlRTQlQkIlODAlRTQlQjklODh8dXNlcmlkOjQ0Om85dDJsdU43Ql9wdE9DTEQxSjRxSzJla1dwQmNAd2VpeGluLnNvaHUuY29tfA; pprdig=XN5aZp6kN_higgOeqKVpNgMP0A0WeqCdz4G-1Tlqh2CZBC2cDFhm9EvSeEuzrOMZw_lIbIWsFSpetlyyviNutRr4lvJWJwg60VGprqv49rUVaKhnGu13QWXjitFyc012AjUxGTcitTQnkFImspm0h3QC3-dnKtPck_w3V-vgPjk; ppinfo=f440c19c8d; passport=5|1638502558|1639712158|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo0ODpneG0lRTQlQkIlOEElRTUlQTQlQTklRTUlOTAlODMlRTQlQkIlODAlRTQlQjklODh8Y3J0OjEwOjE2Mzg1MDI1NTh8cmVmbmljazo0ODpneG0lRTQlQkIlOEElRTUlQTQlQTklRTUlOTAlODMlRTQlQkIlODAlRTQlQjklODh8dXNlcmlkOjQ0Om85dDJsdU43Ql9wdE9DTEQxSjRxSzJla1dwQmNAd2VpeGluLnNvaHUuY29tfA|ad716ac8df|XN5aZp6kN_higgOeqKVpNgMP0A0WeqCdz4G-1Tlqh2CZBC2cDFhm9EvSeEuzrOMZw_lIbIWsFSpetlyyviNutRr4lvJWJwg60VGprqv49rUVaKhnGu13QWXjitFyc012AjUxGTcitTQnkFImspm0h3QC3-dnKtPck_w3V-vgPjk; sgid=05-52078565-AWGpkJ7cBLQlMCNGmoswMLU; JSESSIONID=aaa0USzdlnNHo6KxlCQ1x; ppmdig=1638523489000000d5986846f2dffecaa098713dbeb50232','刘德华','test')            
    main()